{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c2ad8cc2-2daa-425a-b11b-851b0a52bd54",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T06:39:59.977277Z",
     "start_time": "2025-04-25T06:39:59.610757Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np     #只需要下载numpy库即可\n",
    "import random\n",
    "import GridWorld_v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "441bf83e-6a37-45f1-829e-b1affd2194b0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T06:42:45.580068Z",
     "start_time": "2025-04-25T06:42:45.576988Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⬜️⬜️⬜️⬜️⬜️\n",
      "⬜️🚫🚫⬜️⬜️\n",
      "⬜️⬜️🚫⬜️⬜️\n",
      "⬜️🚫✅🚫⬜️\n",
      "⬜️🚫⬜️⬜️⬜️\n",
      "⬆️⬆️⬆️⬆️⬆️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬆️⏫️⬆️⬆️\n",
      "⬆️⏫️✅⏫️⬆️\n",
      "⬆️⏫️⬆️⬆️⬆️\n"
     ]
    }
   ],
   "source": [
    "gamma = 0.9   #折扣因子，越接近0越近视\n",
    "rows = 5      #记得行数和列数这里要同步改\n",
    "columns = 5\n",
    "# gridworld = GridWorld_v1.GridWorld_v1(rows=rows, columns=columns, forbiddenAreaNums=4, targetNums=2, seed = random.randint(1,1000))\n",
    "# gridworld = GridWorld_v1.GridWorld_v1(desc = [\".#\",\".T\"])             #赵老师4-1的例子\n",
    "# gridworld = GridWorld_v1.GridWorld_v1(desc = [\"##.T\",\"...#\",\"....\"])  #随便弄的例子\n",
    "gridworld = GridWorld_v1.GridWorld_v1(forbiddenAreaScore=-10, score=1,desc = [\".....\",\".##..\",\"..#..\",\".#T#.\",\".#...\"]) \n",
    "gridworld.show()\n",
    "\n",
    "value = np.zeros(rows*columns)       #初始化可以任意，也可以全0，这就是state-value\n",
    "qtable = np.zeros((rows*columns,5))  #初始化，这里主要是初始化维数，里面的内容会被覆盖所以无所谓,这里的5是action的数量,这里的qtable就是Q表格,这里是action-value\n",
    "policy = np.argmax(qtable,axis=1)    #初始策略，这里是贪心策略，即选择Q值最大的动作\n",
    "gridworld.showPolicy(policy)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a15f99b",
   "metadata": {},
   "source": [
    "# 网格世界，纵轴是x，横轴是y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2ace6ead-ab02-46c5-b6e7-7deda91a13cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T06:42:48.218990Z",
     "start_time": "2025-04-25T06:42:48.204392Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⬜️⬜️⬜️⬜️⬜️\n",
      "⬜️🚫🚫⬜️⬜️\n",
      "⬜️⬜️🚫⬜️⬜️\n",
      "⬜️🚫✅🚫⬜️\n",
      "⬜️🚫⬜️⬜️⬜️\n",
      "⬆️⬆️⬆️⬆️⬆️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬆️⏫️⬆️⬆️\n",
      "⬆️⏫️✅⏫️⬆️\n",
      "⬆️⏫️⬆️⬆️⬆️\n",
      "Euclidean Distance: 25.0\n",
      "[[0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0.]\n",
      " [0. 1. 1. 1. 0.]\n",
      " [0. 0. 1. 0. 0.]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬅️⏬⬆️⬆️\n",
      "⬆️⏩️✅⏪⬆️\n",
      "⬆️⏩️⬆️➡️⬆️\n",
      "Euclidean Distance: 5.0\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0. ]\n",
      " [0.  0.  1.9 0.  0. ]\n",
      " [0.  1.9 1.9 1.9 0. ]\n",
      " [0.  0.9 1.9 0.9 0. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬅️⏬⬆️⬆️\n",
      "⬆️⏩️✅⏪⬆️\n",
      "⬆️⏩️⬆️⬅️⬆️\n",
      "Euclidean Distance: 5.67\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0. ]\n",
      " [0.  0.  2.7 0.  0. ]\n",
      " [0.  2.7 2.7 2.7 0. ]\n",
      " [0.  1.7 2.7 1.7 0.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬅️⏬⬆️⬆️\n",
      "⬆️⏩️✅⏪⬆️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 5.248800000000001\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0. ]\n",
      " [0.  0.  3.4 0.  0. ]\n",
      " [0.  3.4 3.4 3.4 0.7]\n",
      " [0.  2.4 3.4 2.4 1.5]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬅️⏬⬆️⬆️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 4.782969000000001\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0. ]\n",
      " [0.  0.  4.1 0.  0.7]\n",
      " [0.  4.1 4.1 4.1 1.4]\n",
      " [0.  3.1 4.1 3.1 2.2]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬆️\n",
      "⬆️⬅️⏬⬆️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 4.304672100000002\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0.6]\n",
      " [0.  0.  4.7 0.6 1.2]\n",
      " [0.  4.7 4.7 4.7 2. ]\n",
      " [0.  3.7 4.7 3.7 2.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️⬆️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 4.184141281200001\n",
      "[[0.  0.  0.  0.  0.5]\n",
      " [0.  0.  0.  0.5 1.1]\n",
      " [0.  0.  5.2 1.1 1.8]\n",
      " [0.  5.2 5.2 5.2 2.5]\n",
      " [0.  4.2 5.2 4.2 3.3]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏫️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 3.9540135107340006\n",
      "[[0.  0.  0.  0.5 1. ]\n",
      " [0.  0.  0.5 1.  1.6]\n",
      " [0.  0.  5.7 1.6 2.3]\n",
      " [0.  5.7 5.7 5.7 3. ]\n",
      " [0.  4.7 5.7 4.7 3.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 3.660286792793762\n",
      "[[0.  0.  0.4 0.9 1.4]\n",
      " [0.  0.  0.9 1.4 2. ]\n",
      " [0.  0.  6.1 2.  2.7]\n",
      " [0.  6.1 6.1 6.1 3.4]\n",
      " [0.  5.1 6.1 5.1 4.2]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 3.150134321048129\n",
      "[[0.  0.4 0.8 1.3 1.8]\n",
      " [0.  0.  1.3 1.8 2.4]\n",
      " [0.  0.  6.5 2.4 3.1]\n",
      " [0.  6.5 6.5 6.5 3.8]\n",
      " [0.  5.5 6.5 5.5 4.6]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 2.7017034353459857\n",
      "[[0.3 0.7 1.2 1.6 2.2]\n",
      " [0.  0.3 1.6 2.2 2.8]\n",
      " [0.  0.  6.9 2.8 3.4]\n",
      " [0.  6.9 6.9 6.9 4.2]\n",
      " [0.  5.9 6.9 5.9 5. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 2.4315330918113887\n",
      "[[0.7 1.  1.5 2.  2.5]\n",
      " [0.3 0.7 2.  2.5 3.1]\n",
      " [0.  0.  7.2 3.1 3.7]\n",
      " [0.  7.2 7.2 7.2 4.5]\n",
      " [0.  6.2 7.2 6.2 5.3]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 2.0680188945855833\n",
      "[[0.9 1.3 1.8 2.2 2.8]\n",
      " [0.6 0.9 2.2 2.8 3.4]\n",
      " [0.3 0.  7.5 3.4 4. ]\n",
      " [0.  7.5 7.5 7.5 4.7]\n",
      " [0.  6.5 7.5 6.5 5.6]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 1.754861747691195\n",
      "[[1.2 1.6 2.  2.5 3. ]\n",
      " [0.9 1.2 2.5 3.  3.6]\n",
      " [0.5 0.3 7.7 3.6 4.3]\n",
      " [0.3 7.7 7.7 7.7 5. ]\n",
      " [0.  6.7 7.7 6.7 5.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 1.5506596534144024\n",
      "[[1.4 1.8 2.2 2.7 3.3]\n",
      " [1.1 1.4 2.7 3.3 3.8]\n",
      " [0.8 0.5 7.9 3.8 4.5]\n",
      " [0.5 7.9 7.9 7.9 5.2]\n",
      " [0.2 6.9 7.9 6.9 6. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 1.3083690825684007\n",
      "[[1.6 2.  2.5 2.9 3.5]\n",
      " [1.3 1.6 2.9 3.5 4.1]\n",
      " [1.  0.7 8.1 4.1 4.7]\n",
      " [0.7 8.1 8.1 8.1 5.4]\n",
      " [0.4 7.1 8.1 7.1 6.2]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 1.0597789568804075\n",
      "[[1.8 2.2 2.6 3.1 3.6]\n",
      " [1.5 1.8 3.1 3.6 4.2]\n",
      " [1.2 0.9 8.3 4.2 4.9]\n",
      " [0.9 8.3 8.3 8.3 5.6]\n",
      " [0.6 7.3 8.3 7.3 6.4]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.8584209550731305\n",
      "[[2.  2.4 2.8 3.3 3.8]\n",
      " [1.6 2.  3.3 3.8 4.4]\n",
      " [1.3 1.  8.5 4.4 5.1]\n",
      " [1.  8.5 8.5 8.5 5.8]\n",
      " [0.8 7.5 8.5 7.5 6.6]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.6953209736092333\n",
      "[[2.1 2.5 3.  3.4 4. ]\n",
      " [1.8 2.1 3.4 4.  4.6]\n",
      " [1.5 1.2 8.6 4.6 5.2]\n",
      " [1.2 8.6 8.6 8.6 5.9]\n",
      " [0.9 7.6 8.6 7.6 6.7]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.5632099886234779\n",
      "[[2.3 2.7 3.1 3.6 4.1]\n",
      " [1.9 2.3 3.6 4.1 4.7]\n",
      " [1.6 1.3 8.8 4.7 5.3]\n",
      " [1.3 8.8 8.8 8.8 6.1]\n",
      " [1.1 7.8 8.8 7.8 6.9]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.4562000907850179\n",
      "[[2.4 2.8 3.2 3.7 4.2]\n",
      " [2.  2.4 3.7 4.2 4.8]\n",
      " [1.7 1.4 8.9 4.8 5.5]\n",
      " [1.4 8.9 8.9 8.9 6.2]\n",
      " [1.2 7.9 8.9 7.9 7. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.3695220735358652\n",
      "[[2.5 2.9 3.3 3.8 4.3]\n",
      " [2.2 2.5 3.8 4.3 4.9]\n",
      " [1.8 1.6 9.  4.9 5.6]\n",
      " [1.6 9.  9.  9.  6.3]\n",
      " [1.3 8.  9.  8.  7.1]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.2993128795640511\n",
      "[[2.6 3.  3.4 3.9 4.4]\n",
      " [2.3 2.6 3.9 4.4 5. ]\n",
      " [1.9 1.7 9.1 5.  5.7]\n",
      " [1.7 9.1 9.1 9.1 6.4]\n",
      " [1.4 8.1 9.1 8.1 7.2]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.24244343244688132\n",
      "[[2.7 3.1 3.5 4.  4.5]\n",
      " [2.3 2.7 4.  4.5 5.1]\n",
      " [2.  1.7 9.2 5.1 5.8]\n",
      " [1.7 9.2 9.2 9.2 6.5]\n",
      " [1.5 8.2 9.2 8.2 7.3]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.19637918028197468\n",
      "[[2.8 3.2 3.6 4.1 4.6]\n",
      " [2.4 2.8 4.1 4.6 5.2]\n",
      " [2.1 1.8 9.3 5.2 5.8]\n",
      " [1.8 9.3 9.3 9.3 6.6]\n",
      " [1.6 8.3 9.3 8.3 7.4]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.1590671360283985\n",
      "[[2.8 3.2 3.7 4.1 4.7]\n",
      " [2.5 2.8 4.1 4.7 5.3]\n",
      " [2.2 1.9 9.4 5.3 5.9]\n",
      " [1.9 9.4 9.4 9.4 6.6]\n",
      " [1.6 8.4 9.4 8.4 7.5]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.12884438018300284\n",
      "[[2.9 3.3 3.7 4.2 4.7]\n",
      " [2.6 2.9 4.2 4.7 5.3]\n",
      " [2.2 2.  9.4 5.3 6. ]\n",
      " [2.  9.4 9.4 9.4 6.7]\n",
      " [1.7 8.4 9.4 8.4 7.5]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.104363947948232\n",
      "[[3.  3.4 3.8 4.3 4.8]\n",
      " [2.6 3.  4.3 4.8 5.4]\n",
      " [2.3 2.  9.5 5.4 6. ]\n",
      " [2.  9.5 9.5 9.5 6.8]\n",
      " [1.8 8.5 9.5 8.5 7.6]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.08453479783806828\n",
      "[[3.  3.4 3.8 4.3 4.8]\n",
      " [2.7 3.  4.3 4.8 5.4]\n",
      " [2.4 2.1 9.5 5.4 6.1]\n",
      " [2.1 9.5 9.5 9.5 6.8]\n",
      " [1.8 8.5 9.5 8.5 7.6]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.06847318624883582\n",
      "[[3.1 3.5 3.9 4.4 4.9]\n",
      " [2.7 3.1 4.4 4.9 5.5]\n",
      " [2.4 2.1 9.6 5.5 6.1]\n",
      " [2.1 9.6 9.6 9.6 6.9]\n",
      " [1.9 8.6 9.6 8.6 7.7]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.055463280861557024\n",
      "[[3.1 3.5 3.9 4.4 4.9]\n",
      " [2.8 3.1 4.4 4.9 5.5]\n",
      " [2.4 2.2 9.6 5.5 6.2]\n",
      " [2.2 9.6 9.6 9.6 6.9]\n",
      " [1.9 8.6 9.6 8.6 7.7]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.044925257497860284\n",
      "[[3.1 3.5 4.  4.4 5. ]\n",
      " [2.8 3.1 4.4 5.  5.6]\n",
      " [2.5 2.2 9.7 5.6 6.2]\n",
      " [2.2 9.7 9.7 9.7 6.9]\n",
      " [1.9 8.7 9.7 8.7 7.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.03638945857326765\n",
      "[[3.2 3.6 4.  4.5 5. ]\n",
      " [2.8 3.2 4.5 5.  5.6]\n",
      " [2.5 2.2 9.7 5.6 6.3]\n",
      " [2.2 9.7 9.7 9.7 7. ]\n",
      " [2.  8.7 9.7 8.7 7.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.02947546144434684\n",
      "[[3.2 3.6 4.  4.5 5. ]\n",
      " [2.9 3.2 4.5 5.  5.6]\n",
      " [2.5 2.3 9.7 5.6 6.3]\n",
      " [2.3 9.7 9.7 9.7 7. ]\n",
      " [2.  8.7 9.7 8.7 7.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.02387512376992117\n",
      "[[3.2 3.6 4.1 4.5 5.1]\n",
      " [2.9 3.2 4.5 5.1 5.7]\n",
      " [2.6 2.3 9.7 5.7 6.3]\n",
      " [2.3 9.7 9.7 9.7 7. ]\n",
      " [2.  8.7 9.7 8.7 7.8]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.01933885025363554\n",
      "[[3.3 3.6 4.1 4.6 5.1]\n",
      " [2.9 3.3 4.6 5.1 5.7]\n",
      " [2.6 2.3 9.8 5.7 6.3]\n",
      " [2.3 9.8 9.8 9.8 7.1]\n",
      " [2.1 8.8 9.8 8.8 7.9]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.015664468705444703\n",
      "[[3.3 3.7 4.1 4.6 5.1]\n",
      " [2.9 3.3 4.6 5.1 5.7]\n",
      " [2.6 2.3 9.8 5.7 6.4]\n",
      " [2.3 9.8 9.8 9.8 7.1]\n",
      " [2.1 8.8 9.8 8.8 7.9]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.012688219651410261\n",
      "[[3.3 3.7 4.1 4.6 5.1]\n",
      " [3.  3.3 4.6 5.1 5.7]\n",
      " [2.6 2.4 9.8 5.7 6.4]\n",
      " [2.4 9.8 9.8 9.8 7.1]\n",
      " [2.1 8.8 9.8 8.8 7.9]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.010277457917642266\n",
      "[[3.3 3.7 4.1 4.6 5.2]\n",
      " [3.  3.3 4.6 5.2 5.7]\n",
      " [2.7 2.4 9.8 5.7 6.4]\n",
      " [2.4 9.8 9.8 9.8 7.1]\n",
      " [2.1 8.8 9.8 8.8 7.9]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.008324740913290381\n",
      "[[3.3 3.7 4.2 4.6 5.2]\n",
      " [3.  3.3 4.6 5.2 5.8]\n",
      " [2.7 2.4 9.9 5.8 6.4]\n",
      " [2.4 9.9 9.9 9.9 7.1]\n",
      " [2.1 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0067430401397650785\n",
      "[[3.4 3.7 4.2 4.6 5.2]\n",
      " [3.  3.4 4.6 5.2 5.8]\n",
      " [2.7 2.4 9.9 5.8 6.4]\n",
      " [2.4 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.00546186251320991\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.  3.4 4.7 5.2 5.8]\n",
      " [2.7 2.4 9.9 5.8 6.4]\n",
      " [2.4 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.004424108635700081\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.  3.4 4.7 5.2 5.8]\n",
      " [2.7 2.4 9.9 5.8 6.5]\n",
      " [2.4 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0035835279949169305\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.  3.4 4.7 5.2 5.8]\n",
      " [2.7 2.4 9.9 5.8 6.5]\n",
      " [2.4 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0029026576758826588\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.1 3.4 4.7 5.2 5.8]\n",
      " [2.7 2.5 9.9 5.8 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0023511527174650344\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.1 3.4 4.7 5.2 5.8]\n",
      " [2.7 2.5 9.9 5.8 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0019044337011467018\n",
      "[[3.4 3.8 4.2 4.7 5.2]\n",
      " [3.1 3.4 4.7 5.2 5.8]\n",
      " [2.8 2.5 9.9 5.8 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0015425912979287336\n",
      "[[3.4 3.8 4.2 4.7 5.3]\n",
      " [3.1 3.4 4.7 5.3 5.8]\n",
      " [2.8 2.5 9.9 5.8 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0012494989513224066\n",
      "[[3.4 3.8 4.2 4.7 5.3]\n",
      " [3.1 3.4 4.7 5.3 5.8]\n",
      " [2.8 2.5 9.9 5.8 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "Euclidean Distance: 0.0010120941505710659\n",
      "[[3.4 3.8 4.3 4.7 5.3]\n",
      " [3.1 3.4 4.7 5.3 5.9]\n",
      " [2.8 2.5 9.9 5.9 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "➡️➡️➡️➡️⬇️\n",
      "⬆️⏫️⏩️➡️⬇️\n",
      "⬆️⬅️⏬➡️⬇️\n",
      "⬆️⏩️✅⏪⬇️\n",
      "⬆️⏩️⬆️⬅️⬅️\n",
      "[[3.4 3.8 4.3 4.7 5.3]\n",
      " [3.1 3.4 4.7 5.3 5.9]\n",
      " [2.8 2.5 9.9 5.9 6.5]\n",
      " [2.5 9.9 9.9 9.9 7.2]\n",
      " [2.2 8.9 9.9 8.9 8. ]]\n",
      "[1 1 1 1 2 0 0 1 1 2 0 3 2 1 2 0 1 4 3 2 0 1 0 3 3]\n"
     ]
    }
   ],
   "source": [
    "pre_value = value.copy()+1           #随便弄个不一样的初值\n",
    "\n",
    "gridworld.show()                     #打印gridworld\n",
    "gridworld.showPolicy(policy)         #打印策略\n",
    "\n",
    "cnt = 0                              #防止死循环，不需要也可以（因为可以证明必定收敛）\n",
    "while(np.sum((pre_value-value)**2)>0.001): \n",
    "    print(\"Euclidean Distance:\",np.sum((pre_value-value)**2))  #计算欧几里得距离\n",
    "    pre_value = value.copy()         #保存副本，方便前后对比\n",
    "\n",
    "    cnt = cnt+1                      #防止死循环\n",
    "    if cnt>100:                      \n",
    "        break\n",
    "        \n",
    "    for i in range(rows * columns):  #当前状态Si\n",
    "        nowx = i // columns          #坐标x\n",
    "        nowy = i % columns           #坐标y 但都没用上\n",
    "        \n",
    "        for j in range(5): # 5个action\n",
    "            score,nextState = gridworld.getScore(i,j)        #获取Si状态（25个状态）中，执行动作j后的（得分，下一个状态）\n",
    "            qtable[i][j] = score + gamma * value[nextState]  #开始迭代，更新qtable ,q=r+gamma*v\n",
    "\n",
    "    policy = np.argmax(qtable,axis=1) #这个就是策略了，policy update，但是value-iteration，\n",
    "                                      #因为这里的策略是百分百选argmax的，别的概率是0，所以可以直接忽略策略的不确定性\n",
    "    value = np.max(qtable,axis=1)     #更新value 向量,对应伪代码的value update,因为最大的q值的action就是最优策略，概率为1\n",
    "\n",
    "    print(np.round(value.reshape(rows,columns), 1))\n",
    "    # if cnt==50:\n",
    "    gridworld.showPolicy(policy)\n",
    "print(np.round(value.reshape(rows,columns), 1))\n",
    "print(policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "54a22543-95cb-4e13-a760-624088217550",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⬜️🚫\n",
      "⬜️✅\n",
      "⬆️➡️⬇️⬅️🔄\n",
      "⏫︎⏩️⏪🔄✅\n",
      "⏫\n",
      "︎\n",
      "⏩\n"
     ]
    }
   ],
   "source": [
    "print('⬜️🚫')\n",
    "print('⬜️✅')\n",
    "print('⬆️➡️⬇️⬅️🔄')\n",
    "print('⏫︎⏩️⏪🔄✅')\n",
    "\n",
    "tmp = \"⏫︎⏩️⏬⏪🔄\"\n",
    "print(tmp[0])\n",
    "print(tmp[1])\n",
    "print(tmp[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32a08872-b6b6-4b8a-b552-17ba1251e3f2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
